#!/usr/bin/python
# -*- coding: utf-8 -*-
#encoding=utf-8
'''
Created on 2013年12月9日

@author: daimin

文件处理模块
'''

import comm
import time
import os
import conf


# TEST_MEM_SIZE = 32312312

class FileHandler:
    
    __instance = None
    
    __files = []
    
    def __init__(self, fname, tmpfileprefix=''):
        self.__filename = fname
        self.__logger = comm.getlogger()
        self.__tmpfileprefix = "%s" % (tmpfileprefix)
        
        ### 临时文件目录
        self.__efly_tmp_file_dir = "%s/%s" % (os.path.dirname(__file__) , conf.G_CFG['common']['efly_tmp_file_dir'] )
        self.__file_buffer = -1  ### 默认使用系统缓存
        
        self.__init_file()
        
        
    def get_file_name(self):
        return self.__filename
    
    def __get_line_hash(self, src):
        return (comm.ef_hash(src, self.__hash_factor))  % self.__filenum
        
    def __init_file(self):
        self.__cal_files()
        import random
        self.__hash_factor = random.randint(2, self.__filenum + 1)
        self.__split_file()
        
    def run_func(self, **kwargs):
        """函数的回调,_callfunc即为回调的函数
        """
        it_callfunc = ct_callfunc = af_callfunc = None
        if 'itcall' in kwargs and 'ctcall' in kwargs:
            it_callfunc = kwargs['itcall'] 
            ct_callfunc = kwargs['ctcall'] 
            af_callfunc = kwargs['aftcall'] 
        else:
            self.__logger.error("Function arguments wrong!!!")
            exit(1)
        
        for i in xrange(self.__filenum):
            fh = self.__files[i]
            fh.seek(0)
            it_datas = [it_callfunc(line.strip()) for line in fh if line is not None and line != '']
            ### 一个文件做一次统计处理
            ct_callfunc(it_datas)
            del it_datas
            it_datas = None
            af_callfunc()
            
    def __write_logs(self, i, _str):
        self.__files[i].write(_str)
        
    def __split_file(self):
        if self.__filenum > 1:
            self.__logger.info("Begin split file %s to %s file." % (self.__filename, self.__filenum))
            with open(self.__filename, 'r', self.__file_buffer) as file_handler:
                for line in file_handler:
                    self.__write_logs(self.__get_line_hash(line), line)
            self.__logger.info("End split file %s to %s file." % (self.__filename, self.__filenum))
            
    
    def __empty_tmp_dir(self):
        """清空当前进程的所有文件
        """
        if os.path.exists(self.__efly_tmp_file_dir):
            if os.path.isdir(self.__efly_tmp_file_dir):
                map(lambda x:os.remove('%s/%s' %(self.__efly_tmp_file_dir, x)),\
                     filter(lambda x:x.startswith(self.__tmpfileprefix), os.listdir(self.__efly_tmp_file_dir)))
            else:
                os.remove(self.__efly_tmp_file_dir)
                os.mkdir(self.__efly_tmp_file_dir)
        else:
            os.mkdir(self.__efly_tmp_file_dir)
        
            
    
    def __cal_files(self):
        try:
            avail_mem_size = comm.get_available_mem_size()
            
            #avail_mem_size = TEST_MEM_SIZE
            ## 黄金分割
            avail_mem_size = avail_mem_size * 0.618
            logfsize = os.path.getsize(self.__filename)
            
            size_in_g = logfsize / 1073741824          ### 当前文件大小用G来表示
            self.__file_buffer = -1 if size_in_g < 1 else (size_in_g * 2 + 4) * 1024
            import math
            self.__filenum = 1 if avail_mem_size > logfsize else  int(math.ceil(logfsize / avail_mem_size))

            if self.__filenum > 1:
                ### 分割文件的buffer大小            
                split_file_buff = (self.__file_buffer / self.__filenum) if self.__file_buffer > 0 else -1
                split_file_buff = -1 if split_file_buff < 4096 else split_file_buff
                ### 首先清空临时目录
                self.__empty_tmp_dir()
                self.__files = [open("%s/%s.%d.tmp_%d" % (self.__efly_tmp_file_dir, self.__tmpfileprefix, os.getpid(), i), "a+", split_file_buff) for i in xrange(self.__filenum)]
            else:
                self.__files = [open(self.__filename, "r", self.__file_buffer)]
            return self.__filenum
        except:
            #print e
            pass
        
        return 1
    
    def __del__(self):
        self.__close_files()
    
    def __close_files(self):
        for f in self.__files:
            if f is not None:
                f.close()
                f = None
                
    @staticmethod
    def get_instance(fname, tmpfileprefix):
        FileHandler.__instance = FileHandler(fname, tmpfileprefix) if FileHandler.__instance is None or\
                                                       FileHandler.__instance.get_file_name() != fname else FileHandler.__instance
        return FileHandler.__instance
        



if __name__ == '__main__':
    #print FileHandler("E:\\daiminWorks\\pywork\\bdatademos\\massiveIP.txt").get_file_num()
    start_time = time.clock()
    fh = FileHandler("E:\\daiminWorks\\pywork\\eflyminelog\\bbfile.txt")
    def test(line):
        print line
    def test_count(c):
        print c
    fh.run_func(itcall=test, ctcall=test_count)
    end_time = time.clock()
    comm.getlogger().warn( "######### Parse Success!!! Spend [%f] seconds #################" % (end_time - start_time))
    
    
    
    